{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "from tqdm import tqdm\n",
    "import jsonlines\n",
    "from PIL import Image\n",
    "import concurrent.futures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_jsonl(data_path):\n",
    "    data_list = []\n",
    "    with jsonlines.open(data_path) as reader:\n",
    "        for obj in tqdm(reader):\n",
    "            data_list.append(obj)\n",
    "    return data_list\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "217060it [00:04, 50393.86it/s]\n",
      "100%|██████████| 217060/217060 [00:03<00:00, 60109.40it/s]\n"
     ]
    }
   ],
   "source": [
    "root_dir = Path('/remote-home/share/medical/public/MedICaT')\n",
    "data_path = root_dir / 'release/s2_full_figures_oa_nonroco_combined_medical_top4_public.jsonl'\n",
    "data_list = read_jsonl(data_path)\n",
    "\n",
    "img_path_list = []\n",
    "for obj in tqdm(data_list):\n",
    "    img_path = root_dir / \"release/figures/\" / f\"{obj['pdf_hash']}_{obj['fig_uri']}\"\n",
    "    img_path_list.append(img_path)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 217060/217060 [13:36<00:00, 265.92it/s] \n"
     ]
    }
   ],
   "source": [
    "def broken_img(img_path):\n",
    "    pbar.update(1)\n",
    "    try:\n",
    "        Image.open(img_path)\n",
    "    except:\n",
    "        if len(broken_imgs) % 10 == 0:\n",
    "            print( len(broken_imgs) )\n",
    "        return (img_path, 'broken')\n",
    "    else:\n",
    "        return (img_path, 'OK')\n",
    "\n",
    "broken_imgs = []\n",
    "\n",
    "pbar = tqdm(total=len(img_path_list))\n",
    "\n",
    "# We can use a with statement to ensure threads are cleaned up promptly\n",
    "with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:\n",
    "    # Start the load operations and mark each future with its URL\n",
    "    future_to_url = {executor.submit(broken_img, img_path): img_path for img_path in img_path_list}\n",
    "    for future in concurrent.futures.as_completed(future_to_url):\n",
    "        # url = future_to_url[future]\n",
    "        try:\n",
    "            data = future.result()\n",
    "            broken_imgs.append(data)\n",
    "            # raise ValueError(data)\n",
    "        except:\n",
    "            pass\n",
    "        # except Exception as exc:\n",
    "        #     print('%r generated an exception: %s' % (img_path, exc))\n",
    "        # else:\n",
    "        #     print('%r page is %d bytes' % (url, len(data)))\n",
    "\n",
    "pbar.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "real_broken_imgs = []\n",
    "# for obj in broken_imgs:\n",
    "#     raise ValueError(obj)\n",
    "#     if status == 'broken':\n",
    "#         real_broken_imgs.append(img_path)\n",
    "\n",
    "for obj in broken_imgs:\n",
    "    img_path, status = obj\n",
    "    if status == 'broken':\n",
    "        real_broken_imgs.append(img_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(real_broken_imgs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[PosixPath('/remote-home/share/medical/public/MedICaT/release/figures/ffd83b6453f94f2a1ddb346e324f5bdbf228c1f3_4-Figure3-1.png')]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "real_broken_imgs"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
